# mTurk maker 
################################################################################
# Dependencies
################################################################################
library(data.table)
library(purrr)
library(lubridate)
library(readr)
library(dplyr)
library(readr)
library(readxl)
library(irr)
library(tidyr)
library(ggplot2)
library(tidyverse)
library(tidycomm)
################################################################################
# Setup
################################################################################
rm(list=ls())
# - set dir
args = commandArgs()

scriptName = args[substr(args,1,7) == '--file=']

if (length(scriptName) == 0) {
  scriptName <- rstudioapi::getSourceEditorContext()$path
} else {
  scriptName <- substr(scriptName, 8, nchar(scriptName))
}

pathName = substr(
  scriptName, 
  1, 
  nchar(scriptName) - nchar(strsplit(scriptName, '.*[/|\\]')[[1]][2])
)

setwd(pathName)

emo_vec <- read_file("./emoji_regex_vector.txt")

emojy_replace <- function(string, replacment, emo_vec){
  stringr::str_replace_all(string, emo_vec, replacment)
}
################################################################################
# Relevance
################################################################################
gpt_meysam <- read_csv("../data/chatGPT_data/training_data_relevance_final.csv",
                       col_types = cols(status_id = col_character()))
mturk_mael <- read_csv("../data/mTurk_data/training_data_relevance_final.csv",
                       col_types = cols(status_id = col_character())) 

mturk_mael <- mturk_mael %>% filter(status_id %in% gpt_meysam$status_id) %>% distinct(status_id, .keep_all = T)

#mTurk Results
df <- read_csv("../data/mTurk_data/batch_results_relevance_final.csv", 
               col_types = cols(ApprovalTime = col_character(), 
                                RejectionTime = col_character(), 
                                RequesterFeedback = col_character(),
                                Input.status_id = col_character(),
                                Approve = col_character(),
                                Reject = col_character()))
df <- df %>% filter(is.na(Input.relevant_ra) == F) %>% 
             mutate(Input.relevant_ra = ifelse(Input.relevant_ra == 1, "Relevant", "Irrelevnt"))

# Quality of mTurk Workers
worker_qual <- df %>% filter(Input.relevance_agreement_check == 1) %>% 
                      mutate(worker_vs_ra = ifelse(Answer.category.label == Input.relevant_ra, 1, 0)) %>% 
                      group_by(WorkerId) %>% 
                      summarise(quality = sum(worker_vs_ra)/n(),
                                n_questions = n())

summary(worker_qual$quality)

batch1_real <- df %>% filter(Input.relevance_agreement_check == 1) %>%
  filter(is.na(Answer.category.label) == F) %>% ungroup %>% 
  group_by(Input.status_id)

head(batch1_real$Input.status_id)

batch1_real$pseudo_coder_id <- rep_along(1:nrow(batch1_real), 1:2)
batch1_real <- batch1_real %>% group_by(Input.status_id,pseudo_coder_id) %>% distinct(Input.status_id,pseudo_coder_id, .keep_all = T)

batch1_real <- as_tibble(batch1_real)

# Agreement over all coders...
batch1_real %>% test_icr(unit_var = Input.status_id, coder_var = pseudo_coder_id, Answer.category.label)
batch1_real %>%  tab_frequencies(Answer.category.label)


# Accuracy over all 
batch_accuraacy <- batch1_real %>% select(c(Input.status_id,
                                        Input.relevance_agreement_check,
                                        Input.relevant_ra,Answer.category.label,pseudo_coder_id)) %>%
                               pivot_wider(names_from = c(pseudo_coder_id), values_from = Answer.category.label) %>% 
                               mutate(relevant_mturk = ifelse(`1` == `2`, `1`, FALSE))

batch_accuraacy <- batch_accuraacy %>% mutate(mturk_vs_ra = ifelse(relevant_mturk == Input.relevant_ra, T, F))

batch_accuraacy  %>% group_by(mturk_vs_ra) %>% summarise(n = n()) %>% mutate(f = n/sum(n))

################################################################################
# Problem Solution
################################################################################
mturk_mael <- read_csv("../data/mTurk_data/training_data_problem_solution_final.csv",
                       col_types = cols(status_id = col_character())) 


#mTurk Results
df <- read_csv("../data/mTurk_data/batch_results_problem_solution_final.csv", 
               col_types = cols(ApprovalTime = col_character(), 
                                RejectionTime = col_character(), 
                                RequesterFeedback = col_character(),
                                Input.status_id = col_character(),
                                Approve = col_character(),
                                Reject = col_character()))

df <- df %>% filter(is.na(Input.problem_solution_ra) == F)

unique(df$Input.problem_solution_ra)

# Quality of mTurk Workers
worker_qual <- df %>% 
  mutate(worker_vs_ra = ifelse(Answer.category.label == Input.problem_solution_ra, 1, 0)) %>% 
  group_by(WorkerId) %>% 
  summarise(quality = sum(worker_vs_ra)/n(),
            n_questions = n())

summary(worker_qual$quality)

batch1_real <- df %>%
  filter(is.na(Answer.category.label) == F) %>% ungroup %>% 
  group_by(Input.status_id)

head(batch1_real$Input.status_id)

batch1_real$pseudo_coder_id <- rep_along(1:nrow(batch1_real), 1:2)
batch1_real <- batch1_real %>% group_by(Input.status_id,pseudo_coder_id) %>% distinct(Input.status_id,pseudo_coder_id, .keep_all = T)

batch1_real <- as_tibble(batch1_real)

# Agreement over all coders...
batch1_real %>% test_icr(unit_var = Input.status_id, coder_var = pseudo_coder_id, Answer.category.label)
batch1_real %>%  tab_frequencies(Answer.category.label)


# Accuracy over all 
batch_accuraacy <- batch1_real %>% select(c(Input.status_id,
                                            Input.problem_solution_agreement_check,
                                            Input.problem_solution_ra,Answer.category.label,pseudo_coder_id)) %>%
  pivot_wider(names_from = c(pseudo_coder_id), values_from = Answer.category.label) %>% 
  mutate(relevant_mturk = ifelse(`1` == `2`, `1`, FALSE))

batch_accuraacy <- batch_accuraacy %>% mutate(mturk_vs_ra = ifelse(relevant_mturk == Input.problem_solution_ra, T, F))

batch_accuraacy  %>% group_by(mturk_vs_ra) %>% summarise(n = n()) %>% mutate(f = n/sum(n))
################################################################################
# Frames
################################################################################
mturk_mael <- read_csv("../data/mTurk_data/training_data_frames_final.csv",
                       col_types = cols(status_id = col_character())) 


#mTurk Results
df <- read_csv("../data/mTurk_data/batch_results_frames_final.csv", 
               col_types = cols(ApprovalTime = col_character(), 
                                RejectionTime = col_character(), 
                                RequesterFeedback = col_character(),
                                Input.status_id = col_character(),
                                Approve = col_character(),
                                Reject = col_character()))
df <- df %>% mutate(Answer.category.label = tolower(Answer.category.label))


# Quality of mTurk Workers
worker_qual <- df %>%
  mutate(worker_vs_ra = ifelse(Answer.category.label == Input.frame, 1, 0)) %>% 
  group_by(WorkerId) %>% 
  summarise(quality = sum(worker_vs_ra)/n(),
            n_questions = n())

summary(worker_qual$quality)

batch1_real <- df %>%
  filter(is.na(Answer.category.label) == F) %>% ungroup %>% 
  group_by(Input.status_id)

head(batch1_real$Input.status_id)

batch1_real$pseudo_coder_id <- rep_along(1:nrow(batch1_real), 1:2)
batch1_real <- batch1_real %>% group_by(Input.status_id,pseudo_coder_id) %>% distinct(Input.status_id,pseudo_coder_id, .keep_all = T)

batch1_real <- as_tibble(batch1_real)

# Agreement over all coders...
batch1_real %>% test_icr(unit_var = Input.status_id, coder_var = pseudo_coder_id, Answer.category.label)
batch1_real %>%  tab_frequencies(Answer.category.label)


# Accuracy over all 
batch_accuraacy <- batch1_real %>% select(c(Input.status_id,
                                            Input.frame,Answer.category.label,pseudo_coder_id)) %>%
  pivot_wider(names_from = c(pseudo_coder_id), values_from = Answer.category.label) %>% 
  mutate(relevant_mturk = ifelse(`1` == `2`, `1`, FALSE))

batch_accuraacy <- batch_accuraacy %>% mutate(mturk_vs_ra = ifelse(relevant_mturk == Input.frame, T, F))

batch_accuraacy  %>% group_by(mturk_vs_ra) %>% summarise(n = n()) %>% mutate(f = n/sum(n))

################################################################################
# Stance
################################################################################
mturk_mael <- read_csv("../data/mTurk_data/training_data_stance_final.csv",
                       col_types = cols(status_id = col_character())) 


#mTurk Results
df <- read_csv("../data/mTurk_data/batch_results_stance_final.csv", 
               col_types = cols(ApprovalTime = col_character(), 
                                RejectionTime = col_character(), 
                                RequesterFeedback = col_character(),
                                Input.status_id = col_character(),
                                Approve = col_character(),
                                Reject = col_character()))
df <- df %>% filter(is.na(Input.stance_ra) == F) %>% 
  mutate(Input.stance_ra = ifelse(Input.stance_ra == "Neutral Stance", "NEUTRAL", 
                                  ifelse(Input.stance_ra == "Positive Stance", "POSITIVE",
                                         ifelse(Input.stance_ra == "Negative Stance", "NEGATIVE", "ERROR"))))
         
unique(df$Input.stance_ra)

# Quality of mTurk Workers
worker_qual <- df %>% 
  mutate(worker_vs_ra = ifelse(Answer.category.label == Input.stance_ra, 1, 0)) %>% 
  group_by(WorkerId) %>% 
  summarise(quality = sum(worker_vs_ra)/n(),
            n_questions = n())

summary(worker_qual$quality)

batch1_real <- df %>%
  filter(is.na(Answer.category.label) == F) %>% ungroup %>% 
  group_by(Input.status_id)

head(batch1_real$Input.status_id)

batch1_real$pseudo_coder_id <- rep_along(1:nrow(batch1_real), 1:2)
batch1_real <- batch1_real %>% group_by(Input.status_id,pseudo_coder_id) %>% distinct(Input.status_id,pseudo_coder_id, .keep_all = T)

batch1_real <- as_tibble(batch1_real)

# Agreement over all coders...
batch1_real %>% test_icr(unit_var = Input.status_id, coder_var = pseudo_coder_id, Answer.category.label)
batch1_real %>%  tab_frequencies(Answer.category.label)


# Accuracy over all 
batch_accuraacy <- batch1_real %>% select(c(Input.status_id,
                                            Input.stance_check,
                                            Input.stance_ra,Answer.category.label,pseudo_coder_id)) %>%
  pivot_wider(names_from = c(pseudo_coder_id), values_from = Answer.category.label) %>% 
  mutate(relevant_mturk = ifelse(`1` == `2`, `1`, FALSE))

batch_accuraacy <- batch_accuraacy %>% mutate(mturk_vs_ra = ifelse(relevant_mturk == Input.stance_ra, T, F))

batch_accuraacy  %>% group_by(mturk_vs_ra) %>% summarise(n = n()) %>% mutate(f = n/sum(n))
################################################################################
# Topics
################################################################################
mturk_mael <- read_csv("../data/mTurk_data/training_data_topics_final.csv",
                       col_types = cols(status_id = col_character())) 


#mTurk Results
df <- read_csv("../data/mTurk_data/batch_results_topics_final.csv", 
               col_types = cols(ApprovalTime = col_character(), 
                                RejectionTime = col_character(), 
                                RequesterFeedback = col_character(),
                                Input.status_id = col_character(),
                                Approve = col_character(),
                                Reject = col_character()))
df <- df %>% 
  mutate(Input.topic = ifelse(Input.topic == "general complaint", "COMPLAINTS", 
                              ifelse(Input.topic == "personal complaint", "COMPLAINTS",
                                     ifelse(Input.topic == "platform policies", "PLATFORM POLICIES",
                                            ifelse(Input.topic == "trump ban", "TRUMP BAN", 
                                                   ifelse(Input.topic == "section 230", "SECTION 230", 
                                                          ifelse(Input.topic == "twitter support", "TWITTER SUPPORT",
                                                                 ifelse(is.na(Input.topic) == TRUE, "OTHER", "ERROR")))))))) %>% 
  mutate(Input.topic = replace_na(Input.topic, "OTHER"))

unique(df$Input.topic)

# Quality of mTurk Workers
worker_qual <- df %>% filter(Input.topic_check == T) %>% 
  mutate(worker_vs_ra = ifelse(Answer.category.label == Input.topic, 1, 0)) %>% 
  group_by(WorkerId) %>% 
  summarise(quality = sum(worker_vs_ra)/n(),
            n_questions = n())

summary(worker_qual$quality)

batch1_real <- df %>% filter(Input.topic_check == T) %>% 
  filter(is.na(Answer.category.label) == F) %>% ungroup %>% 
  group_by(Input.status_id)

head(batch1_real$Input.status_id)

batch1_real$pseudo_coder_id <- rep_along(1:nrow(batch1_real), 1:2)
batch1_real <- batch1_real %>% group_by(Input.status_id,pseudo_coder_id) %>% distinct(Input.status_id,pseudo_coder_id, .keep_all = T)

batch1_real <- as_tibble(batch1_real)

# Agreement over all coders...
batch1_real %>% test_icr(unit_var = Input.status_id, coder_var = pseudo_coder_id, Answer.category.label)
batch1_real %>%  tab_frequencies(Answer.category.label)


# Accuracy over all 
batch_accuraacy <- batch1_real %>% select(c(Input.status_id,
                                            Input.topic_check,
                                            Input.topic,Answer.category.label,pseudo_coder_id)) %>%
  pivot_wider(names_from = c(pseudo_coder_id), values_from = Answer.category.label) %>% 
  mutate(relevant_mturk = ifelse(`1` == `2`, `1`, FALSE))

batch_accuraacy <- batch_accuraacy %>% mutate(mturk_vs_ra = ifelse(relevant_mturk == Input.topic, T, F))

batch_accuraacy  %>% group_by(mturk_vs_ra) %>% summarise(n = n()) %>% mutate(f = n/sum(n))
################################################################################